<?php

class JS_Tokenizer {
	
	private $_tokens;
	private $_skip;
	
	public function __construct($file='tokens') {
		$this->_tokens = false;
		$this->_skip = false;
		$this->importTokens($file);
	}
	
	public function getToken($token) {
		return isset($this->_tokens[$token]) ? $this->_tokens[$token] : false;
	}
	
	public function getSkipTokens() {
		return $this->_skip;
	}
	
	public function importTokens($file, $include_path='bin/') {
		$tokens =& json_decode(file_get_contents($include_path . $file), true);
		$this->_tokens =& $tokens['TOKENS'];
		$this->_skip = array('\s', '\n', '\t', '\r', '\f');
	}
	
	public function isSkipToken(&$token) {
		return in_array($token, $this->_skip);
	}
	
}

?>